{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Copyright 2015 Google Inc. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# ==============================================================================\n",
    "\n",
    "\"\"\"Builds the MNIST network.\n",
    "Implements the inference/loss/training pattern for model building.\n",
    "1. inference() - Builds the model as far as is required for running the network\n",
    "forward to make predictions.\n",
    "2. loss() - Adds to the inference model the layers required to generate loss.\n",
    "3. training() - Adds to the loss model the Ops required to generate and\n",
    "apply gradients.\n",
    "This file is used by the various \"fully_connected_*.py\" files and not meant to\n",
    "be run.\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import math\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# The MNIST dataset has 10 classes, representing the digits 0 through 9.\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# The MNIST images are always 28x28 pixels.\n",
    "IMAGE_SIZE = 28\n",
    "IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE\n",
    "\n",
    "\n",
    "def inference(images, hidden1_units, hidden2_units):\n",
    "  \"\"\"Build the MNIST model up to where it may be used for inference.\n",
    "  Args:\n",
    "    images: Images placeholder, from inputs().\n",
    "    hidden1_units: Size of the first hidden layer.\n",
    "    hidden2_units: Size of the second hidden layer.\n",
    "  Returns:\n",
    "    softmax_linear: Output tensor with the computed logits.\n",
    "  \"\"\"\n",
    "  # Hidden 1\n",
    "  with tf.name_scope('hidden1'):\n",
    "    weights = tf.Variable(\n",
    "        tf.truncated_normal([IMAGE_PIXELS, hidden1_units],\n",
    "                            stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),\n",
    "        name='weights')\n",
    "    biases = tf.Variable(tf.zeros([hidden1_units]),\n",
    "                         name='biases')\n",
    "    hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)\n",
    "  # Hidden 2\n",
    "  with tf.name_scope('hidden2'):\n",
    "    weights = tf.Variable(\n",
    "        tf.truncated_normal([hidden1_units, hidden2_units],\n",
    "                            stddev=1.0 / math.sqrt(float(hidden1_units))),\n",
    "        name='weights')\n",
    "    biases = tf.Variable(tf.zeros([hidden2_units]),\n",
    "                         name='biases')\n",
    "    hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases)\n",
    "  # Linear\n",
    "  with tf.name_scope('softmax_linear'):\n",
    "    weights = tf.Variable(\n",
    "        tf.truncated_normal([hidden2_units, NUM_CLASSES],\n",
    "                            stddev=1.0 / math.sqrt(float(hidden2_units))),\n",
    "        name='weights')\n",
    "    biases = tf.Variable(tf.zeros([NUM_CLASSES]),\n",
    "                         name='biases')\n",
    "    logits = tf.matmul(hidden2, weights) + biases\n",
    "  return logits\n",
    "\n",
    "\n",
    "def loss(logits, labels):\n",
    "  \"\"\"Calculates the loss from the logits and the labels.\n",
    "  Args:\n",
    "    logits: Logits tensor, float - [batch_size, NUM_CLASSES].\n",
    "    labels: Labels tensor, int32 - [batch_size].\n",
    "  Returns:\n",
    "    loss: Loss tensor of type float.\n",
    "  \"\"\"\n",
    "  # Convert from sparse integer labels in the range [0, NUM_CLASSES)\n",
    "  # to 1-hot dense float vectors (that is we will have batch_size vectors,\n",
    "  # each with NUM_CLASSES values, all of which are 0.0 except there will\n",
    "  # be a 1.0 in the entry corresponding to the label).\n",
    "  batch_size = tf.size(labels)\n",
    "  labels = tf.expand_dims(labels, 1)\n",
    "  indices = tf.expand_dims(tf.range(0, batch_size), 1)\n",
    "  concated = tf.concat(1, [indices, labels])\n",
    "  onehot_labels = tf.sparse_to_dense(\n",
    "      concated, tf.pack([batch_size, NUM_CLASSES]), 1.0, 0.0)\n",
    "  cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits,\n",
    "                                                          onehot_labels,\n",
    "                                                          name='xentropy')\n",
    "  loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')\n",
    "  return loss\n",
    "\n",
    "\n",
    "def training(loss, learning_rate):\n",
    "  \"\"\"Sets up the training Ops.\n",
    "  Creates a summarizer to track the loss over time in TensorBoard.\n",
    "  Creates an optimizer and applies the gradients to all trainable variables.\n",
    "  The Op returned by this function is what must be passed to the\n",
    "  `sess.run()` call to cause the model to train.\n",
    "  Args:\n",
    "    loss: Loss tensor, from loss().\n",
    "    learning_rate: The learning rate to use for gradient descent.\n",
    "  Returns:\n",
    "    train_op: The Op for training.\n",
    "  \"\"\"\n",
    "  # Add a scalar summary for the snapshot loss.\n",
    "  tf.scalar_summary(loss.op.name, loss)\n",
    "  # Create the gradient descent optimizer with the given learning rate.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "  # Create a variable to track the global step.\n",
    "  global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "  # Use the optimizer to apply the gradients that minimize the loss\n",
    "  # (and also increment the global step counter) as a single training step.\n",
    "  train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "  return train_op\n",
    "\n",
    "\n",
    "def evaluation(logits, labels):\n",
    "  \"\"\"Evaluate the quality of the logits at predicting the label.\n",
    "  Args:\n",
    "    logits: Logits tensor, float - [batch_size, NUM_CLASSES].\n",
    "    labels: Labels tensor, int32 - [batch_size], with values in the\n",
    "      range [0, NUM_CLASSES).\n",
    "  Returns:\n",
    "    A scalar int32 tensor with the number of examples (out of batch_size)\n",
    "    that were predicted correctly.\n",
    "  \"\"\"\n",
    "  # For a classifier model, we can use the in_top_k Op.\n",
    "  # It returns a bool tensor with shape [batch_size] that is true for\n",
    "  # the examples where the label's is was in the top k (here k=1)\n",
    "  # of all logits for that example.\n",
    "  correct = tf.nn.in_top_k(logits, labels, 1)\n",
    "  # Return the number of true entries.\n",
    "  return tf.reduce_sum(tf.cast(correct, tf.int32))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
